#coding: utf8
import urlparse
import time
import os.path
from scrapy import log
import re
from pygeocoder import Geocoder
from decimal import *
from scrapy.http import Request
from scrapy.http import FormRequest
from scrapy.spider import BaseSpider
from scrapy.selector import HtmlXPathSelector
from fresh.items import RestaurantItem
from fresh.items import RestaurantReview
from fresh.AtrapaloMasterDB import AtrapaloMasterDB
import datetime
import MySQLdb
from fresh.utils import Utils
from fresh.AtrapaloUtils import AtrapaloUtils
import sys

class AtrapaloHistSpider(BaseSpider):
    
    handle_httpstatus_list = [404,410] 
    name = "atrapalohist"
    allowed_domains = ["www.atrapalo.com"]
    start_urls = []
    db = AtrapaloMasterDB()
    logLocalName="ATRAPALO_HIST_SCRAPPY"
    numRestFound=0
    
    utils = Utils()
    atrapaloUtils = AtrapaloUtils()

    
    def __init__(self, *args, **kwargs):
        super(AtrapaloHistSpider, self).__init__()
        reload(sys)
        sys.setdefaultencoding('utf-8')
        time = datetime.datetime.now().strftime('%Y%m%d%H%M%S')
        filename = time + '.log'
        filename = os.path.join('logAtrapaloHist/',filename)
        self.utils.logProcessInit(filename)
        
        self.utils.logProcessBegin(self.logLocalName)

        
        self.db.utils=self.utils
        self.atrapaloUtils.utils = self.utils
        
        if self.db.connectDB():
            self.db.readURLs(self.start_urls)
        else:
            raise CloseSpider(reason='Critical Error, can not connect to DB')


    def make_requests_from_url(self, url):
        idRestSource=None
        idRestSource = self.db.getRestFromUrl(url)
        
        
        return Request(url, dont_filter=True, meta = {'idRestSource': idRestSource})
                
    
    def parse(self, response):
        try:               
            hxs = HtmlXPathSelector(response)
            idRestSource = response.meta['idRestSource']
            if idRestSource is None:
                self.utils.logProcessDetail("ERROR DE PROGRAMACIÓN/BD NO SE PUEDE OBTENER EL CODIGO DE RESTAURANTE A PARTIR DE URL: "+str(response.url) , self.utils.ERROR)
                return

            """check if the response is a not found"""
            if response.status in self.handle_httpstatus_list:               
                self.db.setRestaurantStatus(idRestSource,self.db.NACTIVE)
                return

            restaurant = RestaurantItem()
            restaurant['idRestSource'] = idRestSource
            restaurant['url_scrapy'] = response.url

            self.numRestFound=self.numRestFound+1
            self.utils.log("debug","num restaurants found: "+str(self.numRestFound),self.utils.DEBUG)
    
            self.atrapaloUtils.fillRestaurant(response,hxs,restaurant)

            self.atrapaloUtils.fillComments(response,hxs,restaurant)

            self.utils.log("debug","Restaurant: "+ str(restaurant),self.utils.DEBUG)

            return restaurant


        except Exception,e:
            self.utils.logProcessDetail("Error parsing restaurant en ATRAPALO: " + str(restaurant) + "detail: " +str(e) ,self.utils.ERROR)



    def close_spider(self):
        self.utils.logProcessEnd("Robot HIST - ATRAPALO found: "+str(self.numRestFound), self.utils.DEBUG)
